import requests
from bs4 import BeautifulSoup
import math
from pymongo import MongoClient
import urllib.parse
import crawler
import time
import datetime
import time
import re
baseurl='https://www.baidu.com/p/{username}'
#baseurl='http://tieba.baidu.com'


client=MongoClient('localhost',27017)
db=client.cache

def userpop():
    record=db.tiabuser1.find_and_modify(
            query={'status':0},
            update={'$set':{'status':1}}
        )
        # record=self.db.tiabalist.find_one()
    if record:
        return record['_id']
    else:
        return False




if __name__ == '__main__':    
    
    t=int(time.mktime(datetime.datetime.now().timetuple()))

    username=userpop()

    while username:
        try:
            time.sleep(1)
            proxy=crawler.getproxy()
            url=baseurl.format(username=urllib.parse.quote(username))        
            html=requests.get(url,proxies=proxy,timeout=10)
            matchVal = re.search(u"'portrait' : '(?P<portraitVal>.*?)'",html.text)
            
            portraitVal = matchVal.group('portraitVal')

            time.sleep(1)
            proxy=crawler.getproxy()
            htmldetail=requests.get('https://www.baidu.com/p/sys/data/tieba/honor?portrait=%s&t=%s'%(portraitVal,t),proxies=proxy,timeout=10)

            contenval = re.search(u'"tplContent":"(?P<val>.*?)"',htmldetail.text)
            val = contenval.group('val')

            bs=BeautifulSoup(val.replace('\\','').replace('x22','"'),'html.parser')
            tiebalist = bs.find_all(attrs={'class':'honor-item'})
            levellist=bs.find_all(attrs={'class':'honor-level'})

            x=-1
            for l in tiebalist:
                x=x+1
                tiebaname=l.attrs['title']
                level=int(levellist[0].text.strip(' ').replace('级',''))
                
                if l.attrs['title']!='新宋吧':
                    db.tieba_user1.insert({
                        'tieba':tiebaname,
                        'name':username,
                        'level':level,
                        'jingyan':-1,
                        'status':0
                    })
                    print('%s,%s,%s'%(tiebaname,username,level))
                
            db.tiabuser1.update({'_id':username},{'$set':{'status':2}})
            username=userpop()
        except Exception as identifier:
            print('%s,%s'%(username,'error'))
            db.tiabuser1.update({'_id':username},{'$set':{'status':3}})
            username=userpop()